"""猫眼电影爬取"""
import requests
from lxml import etree

BASE_URL = "http://maoyan.com"
HEADERS = {
	"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3554.0 Safari/537.36"
}
my_cookie = dict(ci='1')#设置cookie，从北京读取

def get_detail_urls(url):
	# 具体获取详情url
	rep = requests.get(url=url, headers=HEADERS,cookies=my_cookie)
	html = etree.HTML(rep.text)
	# 找到详情url
	detail_urls = html.xpath("//dl//div[@class='movie-item']/a/@href")
	detail_urls = map(lambda url: BASE_URL+url, detail_urls)
	return detail_urls


def parse_detail_page(url):
	# 获取数据
	movie = {}
	res = requests.get(url=url, headers=HEADERS)
	text = res.content.decode("utf-8")
	html = etree.HTML(text)
	name = html.xpath("//div[@class='movie-brief-container']/h3/text()")[0]
	movie["name"] = name
	lis = html.xpath("//div[@class='movie-brief-container']//li")
	for li in range(len(lis)):
		if li == 0:
			movie["plot"] = lis[li].xpath("./text()")[0]
		if li == 1:
			movie["country"] = lis[li].xpath("./text()")[0].split()[0]
		if li == 2:
			try:
				movie["release_time"] = lis[li].xpath("./text()")[0]
			except Exception as e:
				continue

	avatar = html.xpath("//div[@class='avatar-shadow']/img/@src")
	movie["avatar"] = avatar
	content = html.xpath("//div[@class='mod-content']/span/text()")[0]
	movie["content"] = content
	container = html.xpath("//div[@class='comment-list-container']/ul")
	for li in container:
		li_name = li.xpath(".//span[@class='name']/text()")
		li_content = li.xpath(".//div[@class='comment-content']/text()")
		livs = zip(li_name, li_content)
		movie["user"] = dict((name, value)for name, value in livs)
	return movie


def spider():
	# 获取url自行拼接
	base_url = "http://maoyan.com/films?showType=1&offset={}"
	movies = []
	for i in range(1, 2):
		url = base_url.format(i)
		# 拿到url之后去找到详情页面url
		detail_urls = get_detail_urls(url)
		for detail_url in detail_urls:
			# 去获取详情页面数据
			movie = parse_detail_page(detail_url)
			movies.append(movie)
	print(movies)


if __name__ == '__main__':
	spider()
